# Copyright (c) 2020 Presto Labs Pte. Ltd.
# Author: jhkim

import os
import collections
import numpy
import pandas

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

import coin.strategy.accounting.fee as cfee
from coin.support.pta.logic.pta_stats_calculator import get_futures_multiplier
from coin.tool.strat_monitor.handlers.check_balance_handler import BalanceChecker
from coin.strategy.accounting.pnl_balance_printer.util import is_inverse

SimResponseCompact = collections.namedtuple(
    'SimResponseCompact',
    ['param_label', 'compact_stat', 'miscinfo', 'bbodf', 'product'])


def get_debug_df(combdf, live=False):
  if 'fill_qty' not in combdf:
    combdf['fill_qty'] = 0
    combdf['fill_price'] = 0
    combdf['fill_type'] = 'MAKER'
  cols = [
      'type',
      'timestamp',
      'order_id',
      'sign',
      'fill_type',
      'order_price',
      'order_qty',
      'fill_price',
      'fill_qty',
      'ask0_price',
      'bid0_price',
      'exchange_error_msg',
      'post_only'
  ] + ['order_tag'] * ('order_tag' in combdf.columns)
  combdf = combdf[[col for col in cols if col in combdf.columns]].copy().reset_index(drop=True)
  combdf['timestamp'] = pandas.DatetimeIndex(combdf['timestamp'])
  return combdf


def plot_cumulative_fill_vs_sub(debug_df):
  import coin.pnl.sim_stat_plot as ssplot
  ssplot.setup_plt()
  plt.rcParams["figure.figsize"] = 8, 6
  plt.plot(debug_df['timestamp'],
           debug_df['cum_signed_order_qty'],
           'k-',
           debug_df['timestamp'],
           debug_df['cum_signed_fill_qty'],
           'r-',
           lw=0.5,
           drawstyle='steps-post')
  plt.legend(['signed cum order qty', 'signed cum fill qty'], loc='upper left')
  plt.twinx()
  plt.plot(debug_df['timestamp'], debug_df['order_price'], 'b:', drawstyle='steps-post')
  plt.legend(['order_price'], loc='upper right')


def get_tabulate_str(combdf):
  return combdf.to_string(formatters={'order_id': '{:d}'.format, 'sign': '{:.0f}'.format})
  # slow
  # return tabulate.tabulate(combdf, headers='keys')


def usd_mapping(ccy):
  if ccy in ['BUSD', 'TUSD', 'PAX', 'USDC', 'USDT']:
    return "USD"
  else:
    return ccy


try:
  import numba

  @numba.njit
  def bisect_batch_numba(lhs, rhs):
    i = 0
    j = 0
    rindex = []
    lenlhs = len(lhs)
    lenrhs = len(rhs)
    while i < lenlhs:
      while j < lenrhs:
        if lhs[i] >= rhs[j]:
          j += 1
        else:
          break
      # lhs(order) must not forward look rhs(book)
      rindex.append(max(j - 1, 0))
      i += 1
    return rindex

  def bisect_batch(lhs, rhs):
    return bisect_batch_numba(numpy.array(lhs), numpy.array(rhs))

except ImportError:
  def bisect_batch(lhs, rhs):
    i = 0
    j = 0
    rindex = []
    lenlhs = len(lhs)
    lenrhs = len(rhs)
    while i < lenlhs:
      while j < lenrhs:
        if lhs[i] >= rhs[j]:
          j += 1
        else:
          break
      # lhs(order) must not forward look rhs(book)
      rindex.append(max(j - 1, 0))
      i += 1
    return rindex


def bisect_batch_predet(lhs, rhs, predet_index):
  assert len(lhs) == len(predet_index)
  ret = list(predet_index)
  j = 0
  lenrhs = len(rhs)
  for i, (lhv, predet) in enumerate(zip(lhs, predet_index)):
    j = max(j, predet)
    while j < lenrhs and rhs[j] <= lhv:
      j += 1
    # must not forward look
    ret[i] = max(j - 1, 0)
  return ret


def get_stat_df(sim_response, reserve_map, is_sliced=True, is_pt1m=False, pnl_ccy_override=None):
  orderdf = sim_response.compact_stat.orderdf
  if orderdf is None:
    return None
  miscinfo = sim_response.miscinfo
  bbodf = sim_response.bbodf
  filldf = sim_response.compact_stat.filldf
  product = sim_response.compact_stat.product
  if miscinfo is None:
    initial_pos = 0
    funding_df = pandas.DataFrame()
  else:
    initial_pos = miscinfo.get('initial_pos', 0)
    funding_df = miscinfo.get('funding_df', pandas.DataFrame())

  if 'fill_qty' not in orderdf:
    orderdf['fill_qty'] = 0
    orderdf['fill_price'] = 0
    orderdf['fill_type'] = 'MAKER'

  if funding_df.empty:  # Avoid column not found error
    funding_df = pandas.DataFrame(columns=['timestamp', 'funding_rate'])

  return get_stat_df_from_bbo_order_df_compact(
      product=product,
      initial_pos=initial_pos,
      reserve_map=reserve_map,
      bbodf=bbodf,
      orderdf=orderdf,
      filldf=filldf,
      funding_df=funding_df,
      symbol_info=sim_response.compact_stat.symbol_info,
      squeeze_stat=sim_response.compact_stat.squeeze_stat,
      is_sliced=is_sliced,
      is_orderdf_pt1m=is_pt1m,
      pnl_ccy_override=pnl_ccy_override,)


def get_prcdict(orderdf, bbodf):
  tssbin = []
  prcbin = []
  sgnbin = []
  dotbin = []
  is_tag_mode = "order_tag" in orderdf
  if is_tag_mode:
    orderdf['order_tag'] = orderdf.groupby('order_id')['order_tag'].transform(lambda x: x.ffill())
    tagbin = []
  makermark = 'o'
  takermark = 's'
  canmark = {
      -1: ',',
      1: ',',
  }
  errmark = 'x'
  oidmap = {}
  acc_type = "ORDER_SUBMITTED"
  if "ORDER_ACCEPTED" in orderdf.type.unique():
    acc_type = "ORDER_ACCEPTED"

  for orderrow in orderdf.to_records():
    oid = orderrow['order_id']
    if orderrow.type == acc_type:
      oidmap[oid] = [orderrow.timestamp, orderrow.order_price, orderrow.order_qty]
    elif orderrow.type == 'ORDER_FILLED':
      # add fill mark
      tssbin.append(orderrow.timestamp)
      prcbin.append(orderrow.fill_price)
      sgnbin.append(orderrow.sign)
      if is_tag_mode:
        tagbin.append(orderrow.order_tag)
      if 'TAKER' in orderrow.fill_type:
        dotbin.append(takermark)
      else:
        dotbin.append(makermark)
      if oid in oidmap:
        # add begin and end of submission
        oidmap[oid][2] -= orderrow.fill_qty
        if oidmap[oid][2] == 0:
          tssbin.append(oidmap[oid][0])
          tssbin.append(orderrow.timestamp)
          prcbin.append(orderrow.order_price)
          prcbin.append(numpy.nan)
          sgnbin.append(orderrow.sign)
          sgnbin.append(orderrow.sign)
          dotbin.append(',')
          dotbin.append('')
          if is_tag_mode:
            tagbin.append(orderrow.order_tag)
            tagbin.append(numpy.nan)
          del oidmap[oid]
    elif orderrow.type in ['ORDER_ERROR']:
      tssbin.append(orderrow.timestamp)
      tssbin.append(orderrow.timestamp)
      prcbin.append(orderrow.order_price)
      prcbin.append(numpy.nan)
      sgnbin.append(orderrow.sign)
      sgnbin.append(orderrow.sign)
      dotbin.append(errmark)
      dotbin.append('')
      if is_tag_mode:
        tagbin.append(orderrow.order_tag)
        tagbin.append(numpy.nan)
    elif orderrow.type in ['CANCEL_CONFIRMED', 'ORDER_AUTO_CANCELED']:
      if oid in oidmap:
        tssbin.append(oidmap[oid][0])
        tssbin.append(orderrow.timestamp)
        tssbin.append(orderrow.timestamp)
        prcbin.append(orderrow.order_price)
        prcbin.append(orderrow.order_price)
        prcbin.append(numpy.nan)
        sgnbin.append(orderrow.sign)
        sgnbin.append(orderrow.sign)
        sgnbin.append(orderrow.sign)
        dotbin.append(',')
        dotbin.append(canmark[orderrow.sign])
        dotbin.append('')
        if is_tag_mode:
          tagbin.append(orderrow.order_tag)
          tagbin.append(orderrow.order_tag)
          tagbin.append(numpy.nan)
        oidmap[oid][2] = 0
        del oidmap[oid]
  res = {
      'tssbin': tssbin,
      'prcbin': prcbin,
      'sgnbin': sgnbin,
      'dotbin': dotbin,
  }
  if is_tag_mode:
    res['tagbin'] = tagbin
  return res


def get_stat_df_from_bbo_order_df_compact(
    *,
    product,
    initial_pos,
    reserve_map,
    bbodf,
    orderdf,
    filldf,
    funding_df,
    symbol_info,
    squeeze_stat,
    is_sliced=True,
    is_orderdf_pt1m=False,
    pnl_ccy_override=None,):
  if len(bbodf) > 0:
    bbodf['mid_price'] = (0.5 * (bbodf['ask0_price'] + bbodf['bid0_price'])).astype(float)
    if numpy.isnan(bbodf['mid_price']).sum() > 0:
      bbodf = bbodf.loc[~numpy.isnan(bbodf['mid_price'])].reset_index(drop=True)

  product_relnorm = product.exchange + "-" + (
      product.subscription_symbol
      if hasattr(product, "subscription_symbol")
      else product.symbol)

  orderdf['fill_price_not_na'] = (~orderdf['fill_price'].isna()).astype(float)
  orderdf[['order_price', 'order_qty']] = orderdf[['order_price', 'order_qty']].astype(numpy.float64)
  combdf = orderdf.merge(bbodf.drop_duplicates(), how='outer').merge(funding_df, how='outer').sort_values('timestamp')
  assert ((combdf['fill_qty'] > 0) & (combdf['fill_price'] == 0)).sum() == 0
  has_midp = False
  if 'mid_price' not in combdf:
    combdf['mid_price'] = numpy.nan
  else:
    has_midp = True
  combdf['timestamp'] = combdf['timestamp'].astype(int)
  combdf = combdf.sort_values(['timestamp']).reset_index(drop=True)

  # Fix for cancel accept & cancel error has 0 price in future if it's needed,
  # the actual price can be deduce from ORDER_SUBMIT with same order id
  combdf['order_price'].replace([0.0], numpy.nan, inplace=True)
  mid_price_ts = combdf['timestamp'].copy()
  mid_price_ts.loc[numpy.isnan(combdf['mid_price'])] = numpy.nan
  mid_price_ts = mid_price_ts.fillna(method='ffill')
  combdf['mid_price'].ffill(inplace=True)
  combdf['mtm_price'] = combdf['mid_price']
  if 'ask0_price' in combdf:
    combdf[['ask0_price', 'bid0_price']] = combdf[['ask0_price', 'bid0_price']].astype(numpy.float64)
  #nanmask = numpy.isnan(combdf['mtm_price'])
  #combdf.loc[nanmask, 'mtm_price'] = combdf.loc[nanmask, 'fill_price']
  nanmask = numpy.isnan(combdf['mtm_price'])
  combdf.loc[nanmask, 'mtm_price'] = \
      numpy.where(
          mid_price_ts > combdf['mtm_price_from_order_ts'],
          combdf['mid_price'],
          combdf['mtm_price_from_order'])[nanmask]
  nanmask = numpy.isnan(combdf['mtm_price'])
  combdf.loc[nanmask, 'mtm_price'] = combdf.loc[nanmask, 'fill_price']

  if len(combdf) == 0:
    return None

  if numpy.isnan(combdf['mid_price'].iloc[0]):
    combdf.loc[0, 'mid_price'] = combdf['mtm_price'].iloc[0]
    combdf['mid_price'] = combdf['mid_price'].fillna(method='ffill')

  combdf['fill_qty'] = combdf['fill_qty'].fillna(0)
  combdf['sign'] = combdf['sign'].fillna(0)

  timedret = None
  if has_midp:
    combdf.loc[:combdf['ask0_price'].last_valid_index(), 'ask0_price'].ffill(inplace=True)
    combdf.loc[:combdf['bid0_price'].last_valid_index(), 'bid0_price'].ffill(inplace=True)
    sub = combdf['type'] == 'ORDER_ACCEPTED'
    combdf['payup'] = (
        (combdf['order_price'] - combdf['ask0_price']) * (combdf['sign'] > 0)
        + (combdf['bid0_price'] - combdf['order_price']) * (combdf['sign'] < 0))
    rows = []
    for fwd_ms in [
        -60000, -30000, -10000, -3000, -1000, -100, 0,
        100, 200, 500, 1000, 3000, 5000, 10000, 20000, 40000, 60000, 120000]:
      times = (combdf['timestamp'] + fwd_ms * 1e6).to_numpy()
      idxproj12 = bisect_batch(times, bbodf['timestamp'].astype(int).tolist())
      timediff = (times - bbodf['timestamp'].iloc[idxproj12].to_numpy())
      idxproj12 = numpy.array(idxproj12)
      nan_idx = (idxproj12 < 0) | (timediff > 1e10) | (timediff < 0)
      colname = f'midp_{fwd_ms}ms_fwd'
      combdf[colname] = bbodf['mid_price'].iloc[idxproj12].to_numpy()
      combdf.loc[nan_idx, colname] = numpy.nan
      fill_ret_colname = 'fill_price_ret_bps_%dms_fwd' % fwd_ms
      mid_ret_colname = 'mid_price_ret_bps_%dms_fwd' % fwd_ms
      midps = combdf[colname]
      signs = combdf['sign']
      combdf[fill_ret_colname] = (midps / combdf['fill_price'] - 1) * signs * 1e4
      combdf[mid_ret_colname] = (midps / combdf['mid_price'] - 1) * signs * 1e4
    for category, title, wgtcol, pricecol, idx in [
        (
            'a_taker', '01_SUB_xpostonly(mid2midpret)', 'order_qty', 'mid_price',
            (combdf['post_only'].astype(float) == 0) & (combdf['payup'] >= 0) & sub
        ),
        (
            'a_taker', '02_FILL_taker(filltomidpret)', 'fill_qty', 'fill_price',
            (combdf['fill_type'] == 'TAKER') & (combdf['fill_qty'] > 0)
        ),
        (
            'e_maker', '03_SUB_postonly(mid2midpret)', 'order_qty', 'mid_price',
            (combdf['post_only'].astype(float) > 0) & sub
        ),
        (
            'e_maker', '04_FILL_maker(filltomidpret)', 'fill_qty', 'fill_price',
            (combdf['fill_type'] == 'MAKER') & (combdf['fill_qty'] > 0)
        )]:
      partdf = combdf.loc[idx].reset_index(drop=True)
      for fwd_ms in [
          -60000, -30000, -10000, -3000, -1000, -100, 0,
          100, 200, 500, 1000, 3000, 5000, 10000, 20000, 40000, 60000, 120000]:
        wgt = partdf[wgtcol]
        retvec = partdf[f'{pricecol}_ret_bps_{fwd_ms}ms_fwd']
        isnan = numpy.isnan(retvec)
        if wgt[~isnan].sum() == 0:
          continue
        rows.append({
            'category': category,
            'title': title,
            'fwd(ms)': fwd_ms,
            'wgt': wgt[~isnan].sum(),
            'product': product_relnorm,
            'retbps': numpy.average(retvec[~isnan], weights=wgt[~isnan])
        })
    timedret = pandas.DataFrame(rows)

  if hasattr(product, 'market_name'):
    market_type = product.market_name
  else:
    market_type = product.market_type

  fee_tm = os.environ.get("feebps_tm", None)
  if fee_tm is not None:
    takfee, makfee = fee_tm.split("/")
    takfee = float(takfee) * 1e-4
    makfee = float(makfee) * 1e-4
  else:
    makfee, takfee = cfee.get_fee_rate(
        'Futures' if market_type == 'Futures' else 'Spot',
        product.exchange, product)
  is_inv = is_inverse(product)
  if is_inv:
    orderdf['fill_amt'] = orderdf['fill_qty']
    currency_info = {
        'pnl_currency': usd_mapping(product.base.currency),
        'base_currency': product.base.currency,
        'quote_currency': usd_mapping(product.quote.currency)
    }
  else:
    pnl_currency = 'BTC' if product.exchange == "Bitmex" else usd_mapping(product.quote.currency)
    orderdf['fill_amt'] = orderdf['fill_price'].fillna(0) * orderdf['fill_qty']
    currency_info = {
        'pnl_currency': pnl_currency,
        'base_currency': product.base.currency,
        'quote_currency': usd_mapping(product.quote.currency)
    }

  multiplier = 1

  if market_type == 'Futures':
    first_dt = pandas.DatetimeIndex([combdf['timestamp'].iloc[0]]).to_pydatetime()[0]
    multiplier = get_futures_multiplier('Futures', product.exchange, product.symbol, first_dt)
  elif pnl_ccy_override is not None and usd_mapping(product.quote.currency) != pnl_ccy_override:
    currency_info = {
        'pnl_currency': pnl_ccy_override,
        'base_currency': product.base.currency,
        'quote_currency': pnl_ccy_override
    }
    multiplier = BalanceChecker.query_exchange_rates_by_dates(
        usd_mapping(product.quote.currency),
        pnl_ccy_override,
        pandas.to_datetime(combdf['timestamp']).dt.date.drop_duplicates().tolist())
    multiplier = pandas.to_datetime(combdf['timestamp']).dt.date.map(multiplier)

  nanqty = numpy.isnan(orderdf['order_qty'])
  orderdf.loc[nanqty, 'order_qty'] = orderdf.loc[nanqty, 'fill_qty']
  if is_inv:
    combdf['fill_amt'] = combdf['fill_qty']
    orderdf['sub_amt'] = orderdf['order_qty']
  else:
    combdf['fill_amt'] = combdf['fill_price'].fillna(0) * combdf['fill_qty']
    orderdf['sub_amt'] = orderdf['order_price'].ffill().bfill() * orderdf['order_qty']

  combdf['open_pos'] = initial_pos + (combdf['fill_qty'] * combdf['sign']).cumsum()

  if not is_inv:
    combdf['fill_price_not_na'] = 1
  combdf['fill_amt'] *= combdf['fill_price_not_na']

  makidx = combdf['fill_type'] == 'MAKER'
  combdf['makfee'] = combdf['fill_amt'] * makidx * makfee
  combdf['takfee'] = combdf['fill_amt'] * ~makidx * takfee
  combdf['maker_amt'] = combdf['fill_amt'] * makidx
  combdf['taker_amt'] = combdf['fill_amt'] * ~makidx

  def inv_pnl_each(fill_price, fill_avg_price, position, multiplier):
    if fill_avg_price == 0:
      ret = 0
    else:
      ret = (fill_price / fill_avg_price - 1) * (position) / fill_price * multiplier
    return ret

  if is_inv:
    position = initial_pos
    fill_avg_price = combdf['mtm_price'].iloc[0]
    fill_avg_price_list = []
    realized_pnl_list = []
    realized_pnl = 0
    fills = combdf[combdf['fill_qty'] != 0][['fill_price', 'fill_qty', 'sign']]
    combdf.loc[combdf['open_pos'].abs() < 1e-6, 'open_pos'] = 0
    assert multiplier is not None
    for (_, fill_price, fill_qty, sign) in fills.itertuples():
      # hack: to fix virtual fill issues!
      if numpy.isnan(fill_price):
        fill_price = fill_avg_price
      new_position = position + fill_qty * sign
      if fill_avg_price == 0 and fill_price != 0:
        fill_avg_price = fill_price
      if new_position == 0:
        if position != 0:
          realized_pnl += inv_pnl_each(fill_price, fill_avg_price, position, multiplier)
        fill_avg_price = 0
        realized_pnl_list.append(realized_pnl)
        fill_avg_price_list.append(fill_avg_price)
      else:
        if position != 0:
          if numpy.sign(new_position) != numpy.sign(position):
            # prev position is fully filled
            realized_pnl += inv_pnl_each(fill_price, fill_avg_price, position, multiplier)
            new_fill_avg_price = fill_price
          elif numpy.sign(position) == sign:
            # add water
            if fill_avg_price == 0:
              new_fill_avg_price = fill_avg_price
            else:
              new_fill_avg_price = (fill_avg_price * position
                                    + fill_price * fill_qty * sign) / new_position
          else:
            # prev position is partially filled
            new_fill_avg_price = fill_avg_price
            if fill_avg_price != 0:
              realized_pnl += inv_pnl_each(fill_price, fill_avg_price, -fill_qty * sign, multiplier)
        else:
          new_fill_avg_price = fill_price
        fill_avg_price = numpy.nan_to_num(new_fill_avg_price)
        realized_pnl_list.append(realized_pnl)
        fill_avg_price_list.append(fill_avg_price)
      position = new_position
    combdf.loc[combdf['fill_qty'] != 0, 'realized_pnl'] = realized_pnl_list
    combdf.loc[combdf['fill_qty'] != 0, 'fill_avg_price'] = fill_avg_price_list
    combdf[['realized_pnl', 'fill_avg_price']] = combdf[[
        'realized_pnl', 'fill_avg_price']].fillna(method='ffill')
    mtm_price = numpy.where(~numpy.isnan(combdf['mtm_price']),
                            combdf['mtm_price'],
                            combdf['fill_price'].fillna(method='ffill'))
    mtm_price = pandas.Series(mtm_price).fillna(method='ffill').to_numpy()
    combdf['unrealized_pnl'] = (
        mtm_price / combdf['fill_avg_price'] - 1) * (
            combdf['open_pos']) / mtm_price * multiplier
    combdf.loc[combdf['fill_avg_price'] == 0, 'unrealized_pnl'] = 0
    combdf.loc[combdf['open_pos'] == 0, 'unrealized_pnl'] = 0
    combdf['pnl_gross'] = combdf['unrealized_pnl'] + combdf['realized_pnl']
    combdf['pnl_gross'] = combdf['pnl_gross'].fillna(method='ffill')
    combdf['makfee'] *= multiplier / combdf['fill_price']
    combdf['takfee'] *= multiplier / combdf['fill_price']
    combdf['makfee'] = combdf['makfee'].fillna(0)
    combdf['takfee'] = combdf['takfee'].fillna(0)
    combdf['fill_cash'] = combdf['fill_amt'] * multiplier
    combdf['open_cash'] = combdf['open_pos'] * multiplier
    combdf['open_cash2'] = combdf['open_cash'] / mtm_price
    combdf['funding_fee'] = combdf['funding_rate'].fillna(0) * combdf['open_cash'] / mtm_price
  else:
    reserve_map[product] = reserve_map.get(product, 0)
    mtmps = combdf['mtm_price'].fillna(method='ffill').fillna(method='bfill')
    mtmp = mtmps.iloc[0]
    adj_reserve_close_pq = reserve_map[product] * mtmp
    combdf['close_pq'] = (-combdf['fill_amt'] * combdf['sign']).cumsum() + adj_reserve_close_pq
    combdf['open_pq'] = ((combdf['open_pos'] - reserve_map[product]) * mtmps)
    combdf['close_pq'] += -initial_pos * mtmp
    combdf['pnl_gross'] = combdf['close_pq'] + combdf['open_pq']
    combdf['pnl_gross'] *= multiplier
    combdf['makfee'] *= multiplier
    combdf['takfee'] *= multiplier
    combdf['fill_cash'] = combdf['fill_amt'] * multiplier
    combdf['open_cash'] = combdf['open_pq'] * multiplier
    combdf['open_cash2'] = combdf['open_cash']
    combdf['funding_fee'] = combdf['funding_rate'].fillna(0) * combdf['open_cash']

  combdf['fill_price'] = combdf['fill_price'].fillna(0)
  # virtual fills adj
  pnl_gross_diff = combdf['pnl_gross'].diff()
  is_virtual_fill = ((combdf['fill_qty'] > 0) & numpy.isnan(combdf['order_price']))
  pnl_gross_diff[(pnl_gross_diff != 0) & is_virtual_fill] = 0
  combdf['pnl_gross'] = pnl_gross_diff.cumsum()

  if os.environ.get('pplot_dbg', '0') == '1':
    plt.subplot(311)
    if is_inv:
      plt.plot(pandas.DatetimeIndex(combdf['timestamp']),combdf['unrealized_pnl'],lw=0.5)
      plt.plot(pandas.DatetimeIndex(combdf['timestamp']),combdf['realized_pnl'],lw=0.5)
      plt.legend(['unreal','real'])
    else:
      plt.plot(pandas.DatetimeIndex(combdf['timestamp']),combdf['open_pq'],lw=0.5)
      plt.plot(pandas.DatetimeIndex(combdf['timestamp']),combdf['close_pq'],lw=0.5)
      plt.legend(['open','cls'])
    plt.subplot(312)
    plt.plot(pandas.DatetimeIndex(combdf['timestamp']),combdf['mtm_price'],lw=0.5)
    plt.plot(pandas.DatetimeIndex(combdf['timestamp']),combdf['order_price'],lw=0.5)
    plt.plot(pandas.DatetimeIndex(combdf['timestamp']),combdf['mtm_price_from_order'],lw=0.5)
    plt.subplot(313)
    plt.plot(pandas.DatetimeIndex(combdf['timestamp']),combdf['pnl_gross'],lw=0.5)
    plt.savefig(f'pplot_result/test_{product.symbol}_{os.environ.get("use_bbo")}.png')
    plt.close()

  combdf['pnl_net'] = (
      combdf['pnl_gross']
      - combdf['makfee'].cumsum()
      - combdf['takfee'].cumsum()
      - combdf['funding_fee'].cumsum())
  orderdf['timestamp'] = orderdf['timestamp'].astype(int)
  orderdf = orderdf.sort_values('timestamp').reset_index(drop=True)

  accdf = orderdf.loc[(orderdf['type'] == "ORDER_ACCEPTED")].reset_index(drop=True)
  sub_qty_sum = accdf['order_qty'].sum()
  avg_order_qty = accdf['order_qty'].mean()
  taker_qty_sum = (orderdf['fill_qty'] * (orderdf['fill_type'] == 'TAKER')).sum()
  maker_qty_sum = (orderdf['fill_qty'] * (orderdf['fill_type'] == 'MAKER')).sum()
  accdf = orderdf.loc[
      (orderdf['type'] == "ORDER_ACCEPTED")].reset_index(drop=True)
  if is_inv:
    acc_order_amt = accdf['order_qty']
  else:
    acc_order_amt = accdf['order_qty'] * accdf['order_price']

  accsign = orderdf.loc[orderdf['tag'] == "ORDER_ACCEPTED"]['sign']

  order_count = orderdf['order_id'].unique().shape[0]
  if order_count == 0:
    order_count = symbol_info.order_count

  fill_stat = collections.OrderedDict()
  fill_stat.update({
      'sub_count': order_count,
      'sub_amt':
          acc_order_amt.sum(),
      'mak_amt': (
          (orderdf['fill_type'] == 'MAKER') * orderdf['fill_amt'] * orderdf['fill_price_not_na']).sum(),
      'tak_amt': (
          (orderdf['fill_type'] == 'TAKER') * orderdf['fill_amt'] * orderdf['fill_price_not_na']).sum(),
      'currency_info': currency_info,
      'buy_ack_count': (accsign == 1).sum(),
      'sell_ack_count': (accsign == -1).sum(),
      'buy_can_count': (orderdf.loc[orderdf['tag'] == 'CAN']['sign'] == 1).sum(),
      'sell_can_count': (orderdf.loc[orderdf['tag'] == 'CAN']['sign'] == -1).sum()
  })
  if 'trade_qty_interval' in bbodf:
    if is_inv:
      market_volume_amt = bbodf['trade_qty_interval'].sum()
    else:
      market_volume_amt = bbodf['trade_qty_interval'].sum() * mtmp
    fill_stat['market_volume_amt'] = market_volume_amt
  else:
    fill_stat['market_volume_amt'] = numpy.nan

  fill_stat['order_count'] = orderdf['order_id'].unique().shape[0]
  fill_stat['pnl_net'] = combdf['pnl_net'].iloc[-1]
  fill_stat['pnl_gross'] = combdf['pnl_gross'].iloc[-1]
  fill_stat['pnl_fee'] = combdf['makfee'].sum() + combdf['takfee'].sum()
  fill_stat['funding_fee'] = combdf['funding_fee'].sum()
  fill_stat['fill_amt'] = combdf['fill_amt'].sum()
  fill_stat['taker_amt'] = combdf['taker_amt'].sum()
  fill_stat['maker_amt'] = combdf['maker_amt'].sum()
  # note: better to be nan than wrongly calculated
  fill_stat['net_ret'] = numpy.nan
  fill_stat['grs_ret'] = numpy.nan
  fill_stat['sharpe'] = numpy.nan
  if isinstance(multiplier, pandas.Series):
    # multiplier is later used to adjust for market volume, should be ok to not exact by date
    multiplier = multiplier.iloc[0]
  fill_stat['multiplier'] = multiplier
  if is_orderdf_pt1m:
    fill_stat['peak_sub_count'] = orderdf['weight'].max()
    fill_stat['sub_count'] = orderdf['weight'].sum()
    fill_stat['sub_amt_filled'] = orderdf['sub_amt_filled'].sum()
  else:
    submit_df = orderdf[orderdf['tag'] == 'ORDER_SUBMITTED']
    submit_df['timestamp_dt'] = pandas.to_datetime(submit_df['timestamp'], unit='ns')
    count_per_min = submit_df.groupby(pandas.Grouper(key='timestamp_dt', freq='1T')).size().reset_index(name='count')
    fill_stat['peak_sub_count'] = count_per_min.max()['count']
    fill_stat['sub_count'] = orderdf['order_id'].unique().shape[0]
    def groupby_first_fast(df, groupby):
      # Get rid of missing values that break nth.
      df = df[~(df[groupby].isnull())]
      # Use nth instead of first for speed.
      return df.groupby(groupby).nth(0)

    uoiq = groupby_first_fast(
        orderdf[['order_id', 'order_price', 'sub_amt']].loc[orderdf['tag'] == 'ORDER_FILLED'],
        'order_id')
    uoiq = uoiq[~numpy.isnan(uoiq['order_price'])]
    fill_stat['sub_amt_filled'] = numpy.nansum(uoiq['sub_amt'])
    # assert fill_stat['sub_amt_filled'] > fill_stat['fill_amt']

  fill_stat['fill_count'] = (
      orderdf['order_id'].loc[orderdf['tag'] == 'ORDER_FILLED']).unique().shape[0]

  if squeeze_stat and hasattr(symbol_info, "order_count") and symbol_info.order_count > 0:
    fill_stat['sub_count'] = symbol_info.order_count
    fill_stat['fill_count'] = symbol_info.fill_count
    # import pdb; pdb.set_trace()
    if is_inv:
      fill_stat['sub_amt'] = symbol_info.order_qty
      fill_stat['fill_amt'] = symbol_info.fill_qty
      fill_stat['maker_amt'] = symbol_info.fill_qty_maker
      fill_stat['taker_amt'] = symbol_info.fill_qty_taker
      fill_stat['fill_amt_full'] = symbol_info.fill_qty_full
      fill_stat['sub_amt_filled'] = symbol_info.order_qty_filled
    else:
      fill_stat['sub_amt'] = symbol_info.order_pq
      fill_stat['fill_amt'] = symbol_info.fill_pq
      fill_stat['maker_amt'] = symbol_info.fill_pq_maker
      fill_stat['taker_amt'] = symbol_info.fill_pq_taker
      fill_stat['fill_amt_full'] = symbol_info.fill_pq_full
      fill_stat['sub_amt_filled'] = symbol_info.order_pq_filled

  pos_magnitude = combdf['open_cash2'].max() - combdf['open_cash2'].min()
  if market_type == 'Futures':
    if product.exchange == "Bitflyer":
      leverage_ratio = 2
    else:
      leverage_ratio = 20
  else:
    leverage_ratio = 1
  fill_stat['leverage_ratio'] = leverage_ratio
  fill_stat['required_margin'] = pos_magnitude / leverage_ratio
  possign = numpy.sign(combdf['open_pos'])
  uniqsign = possign[possign.ne(possign.shift())]
  fill_stat['flip_count'] = (uniqsign != 0).sum()
  if sub_qty_sum == 0:
    fill_stat['fill_ratio'] = 0
    fill_stat['taker_ratio'] = 0
    fill_stat['maker_ratio'] = 0
  else:
    fill_stat['fill_ratio'] = (taker_qty_sum + maker_qty_sum) / sub_qty_sum
    fill_stat['taker_ratio'] = taker_qty_sum / sub_qty_sum
    fill_stat['maker_ratio'] = maker_qty_sum / sub_qty_sum
  pnl_stat = fill_stat.copy()
  no_ce_df = orderdf.loc[orderdf.type != 'CANCEL_ERROR']
  no_ce_df_groupby_id = no_ce_df[['order_id', 'timestamp']].groupby('order_id')
  pnl_stat['order_avg_lifetime'] = (no_ce_df_groupby_id.max()['timestamp']
                                    - no_ce_df_groupby_id.min()['timestamp']).mean()
  fill_stat['avg_order_qty'] = avg_order_qty
  group_stat = collections.defaultdict(float)
  orderdf = orderdf.sort_values(['timestamp', 'type']).reset_index(drop=True)
  prcdict = get_prcdict(orderdf, bbodf)
  transcnt = (orderdf.type == "ORDER_ACCEPTED").sum() + (orderdf.type == 'CANCEL_CONFIRMED').sum()
  tperiod = orderdf['timestamp'].astype(int).max() - orderdf['timestamp'].astype(int).min()
  group_stat['tps'] = transcnt / tperiod * 1e9
  group_stat['duration'] = tperiod
  pnl_stat['duration'] = tperiod

  if len(bbodf) > 0:
    combdf['bid0_price'] = combdf['bid0_price'].fillna(method='ffill')
    combdf['ask0_price'] = combdf['ask0_price'].fillna(method='ffill')
  else:
    combdf['bid0_price'] = numpy.nan
    combdf['ask0_price'] = numpy.nan

  pnl_cols = [
      'timestamp', 'bid0_price', 'ask0_price', 'pnl_net', 'order_id', 'mtm_price',
      'open_pos', 'open_cash', 'fill_cash', 'fill_qty', 'fill_price',
      'order_price', 'order_qty', 'sign', 'type', 'fill_type', 'post_only'
  ] + ["submit_max_rate_used_ratio"] * ("submit_max_rate_used_ratio" in combdf.columns
  ) + ["cancel_max_rate_used_ratio"] * ("cancel_max_rate_used_ratio" in combdf.columns)

  for col in ["submit_max_rate_used_ratio", "cancel_max_rate_used_ratio"]:
    if col in combdf.columns:
      combdf[col] = combdf[col].ffill()

  order_cols = [
      'timestamp', 'order_qty', 'fill_qty', 'fill_type', 'type', 'order_id', 'order_price',
      'fill_price', 'sign'
  ] + ['order_tag'] * ('order_tag' in orderdf.columns)

  combdf = combdf.loc[
      (combdf['pnl_net'].diff() != 0)
      | ((combdf['open_cash'].abs() >= 1e-2) & (combdf.index % 100 == 0))
      | (combdf.index >= combdf.shape[0] - 1)] if is_sliced else combdf
  assert (~numpy.isnan(combdf['timestamp'])).all()

  return {
      'combdf': combdf[pnl_cols],
      'orderdf': orderdf[order_cols],
      'pnl_stat': pnl_stat,
      'fill_stat': fill_stat,
      'group_stat': group_stat,
      'currency_info': currency_info,
      'timedret': timedret,
      'prcdict': prcdict,
      'symbol_info': symbol_info,
  }
